////////////////////////////////////////////////////////////////////////////
//
//  Crytek Engine Source File.
//  Copyright (C), Crytek Studios, 2002.
// -------------------------------------------------------------------------
//  File name:   statobjman.cpp
//  Version:     v1.00
//  Created:     28/5/2001 by Vladimir Kajalin
//  Compilers:   Visual Studio.NET
//  Description: Loading trees, buildings, ragister/unregister entities for rendering
// -------------------------------------------------------------------------
//  History:
//
////////////////////////////////////////////////////////////////////////////

#include "StdAfx.h"

#include "StatObj.h"
#include "ObjMan.h"
#include "VisAreas.h"
#include "terrain_sector.h"
#include "CullBuffer.h"
#include "3dEngine.h"
#include "IndexedMesh.h"
#include "WaterVolumes.h"
#include "Brush.h"
#include "ICryAnimation.h"
#include "DecalRenderNode.h"
#include "FogVolumeRenderNode.h"
#include "RoadRenderNode.h"

int CObjManager::m_nUpdateStreamingPrioriryRoundId = 1;
int CObjManager::m_nUpdateStreamingPrioriryRoundIdFast = 1;
int CObjManager::s_nLastStreamingMemoryUsage = 0;


struct CObjManager_Cmp_Streamable_Priority
{
	
	// returns true if v1 < v2
	ILINE bool operator()( const SStreamAbleObject &v1, const SStreamAbleObject &v2) const
	{
		IStreamable * arrObj[2] = { v1.GetStreamAbleObject(), v2.GetStreamAbleObject() };

		// compare priorities
		if(v1.fCurImportance > v2.fCurImportance)
			return true;
		if(v1.fCurImportance < v2.fCurImportance)
			return false;

		// give low lod's and small meshes higher priority
		int MemUsage0 = v1.GetStreamableContentMemoryUsage();
		int MemUsage1 = v2.GetStreamableContentMemoryUsage();
		if(MemUsage0 < MemUsage1)
			return true;
		if(MemUsage0 > MemUsage1)
			return false;

		// fix sorting consistency
		if(arrObj[0] > arrObj[1])
			return true;
		if(arrObj[0] < arrObj[1])
			return false;

		//[AlexMc|04.02.10] This assert isn't correct if we use quicksort (we might compare the pivot against itself)
		//assert(!"Duplicated object found");

		return false;
	}
};

void CObjManager::RegisterForStreaming(IStreamable*pObj) 
{ 
	SStreamAbleObject streamAbleObject(pObj);
  if(m_arrStreamableObjects.Find(streamAbleObject)<0) 
    m_arrStreamableObjects.Add(streamAbleObject);  
}

void CObjManager::UnregisterForStreaming(IStreamable*pObj) 
{ 
	if (m_arrStreamableObjects.size() > 0)
	{
		SStreamAbleObject streamAbleObject(pObj, false);
		m_arrStreamableObjects.Delete(streamAbleObject);

		if (m_arrStreamableObjects.empty())
			stl::free_container(m_arrStreamableObjects);
	}
}

void CObjManager::UpdateObjectsStreamingPriority(bool bSyncLoad)
{
  FUNCTION_PROFILER_3DENGINE;

  if(bSyncLoad)
    PrintMessage("Updating level streaming priorities for camera position (%d,%d,%d) (LevelFrameId = %d)", 
      (int)m_vStreamPreCachePoint.x, (int)m_vStreamPreCachePoint.y, (int)m_vStreamPreCachePoint.z, Get3DEngine()->GetFramesSinceLevelStart());

	CVisAreaManager *pVisAreaMgr = GetVisAreaManager();
	CVisArea * pCurArea = pVisAreaMgr ? (CVisArea *)pVisAreaMgr->GetVisAreaFromPos(m_vStreamPreCachePoint) : NULL;

  if(bSyncLoad || (GetFrameID()&3) || GetFloatCVar(e_StreamCgfFastUpdateMaxDistance)==0)
  {
    for(int nPass=0; nPass<(1+2*(int)bSyncLoad); nPass++)
    {
      if(!m_arrUpdateStreamingPrioriryStack.Count())
      {
        FRAME_PROFILER( "UpdateObjectsStreamingPriority_Init", GetSystem(), PROFILE_3DENGINE );

        m_nUpdateStreamingPrioriryRoundId ++;

        if(GetCVars()->e_StreamCgf==2)
          PrintMessage("UpdateObjectsStreamingPriority_Restart %d", GetFrameID());

        if(CVisArea *pRoot0 = pCurArea)
        {
          m_tmpAreas0.Clear();
          pRoot0->AddConnectedAreas(m_tmpAreas0, GetCVars()->e_StreamPredictionMaxVisAreaRecursion);

					bool bFoundOutside = false;

					if (!GetCVars()->e_StreamPredictionAlwaysIncludeOutside)
					{
						for(int v=0; v<m_tmpAreas0.Count(); v++)
						{
							CVisArea * pN1 = m_tmpAreas0[v];
							if (pN1->IsPortal() && pN1->m_lstConnections.Count() == 1)
							{
								bFoundOutside = true;
								break;
							}
						}
					}
					else
					{
						bFoundOutside = true;
					}

					if (bFoundOutside)
					{
						for(int nSID=0; nSID<Get3DEngine()->m_pObjectsTree.Count(); nSID++)
				      if(Get3DEngine()->IsSegmentSafeToUse(nSID) && Get3DEngine()->m_pObjectsTree[nSID])
								m_arrUpdateStreamingPrioriryStack.Add(Get3DEngine()->m_pObjectsTree[nSID]);
					}

          for(int v=0; v<m_tmpAreas0.Count(); v++)
          {
            CVisArea * pN1 = m_tmpAreas0[v];
            assert(m_arrUpdateStreamingPrioriryStack.Find(pN1->m_pObjectsTree)<0);
            if(pN1->m_pObjectsTree)
              m_arrUpdateStreamingPrioriryStack.Add(pN1->m_pObjectsTree);
          }
        }
        else if(GetVisAreaManager())
        {
					for(int nSID=0; nSID<Get3DEngine()->m_pObjectsTree.Count(); nSID++)
			      if(Get3DEngine()->IsSegmentSafeToUse(nSID) && Get3DEngine()->m_pObjectsTree[nSID])
							m_arrUpdateStreamingPrioriryStack.Add(Get3DEngine()->m_pObjectsTree[nSID]);

          // find portals around
          m_tmpAreas0.Clear();
          GetVisAreaManager()->MakeActiveEntransePortalsList(NULL, m_tmpAreas0, NULL);

          // make list of areas for streaming
          m_tmpAreas1.Clear();
          for(int p=0; p<m_tmpAreas0.Count(); p++)
            if(CVisArea * pRoot = m_tmpAreas0[p])
              pRoot->AddConnectedAreas(m_tmpAreas1, GetCVars()->e_StreamPredictionMaxVisAreaRecursion);

          // fill list of object trees
          for(int v=0; v<m_tmpAreas1.Count(); v++)
          {
            CVisArea * pN1 = m_tmpAreas1[v];
            assert(m_arrUpdateStreamingPrioriryStack.Find(pN1->m_pObjectsTree)<0);
            if(pN1->m_pObjectsTree)
              m_arrUpdateStreamingPrioriryStack.Add(pN1->m_pObjectsTree);
          }
        }
				else
				{
					for(int nSID=0; nSID<Get3DEngine()->m_pObjectsTree.Count(); nSID++)
			      if(Get3DEngine()->IsSegmentSafeToUse(nSID) && Get3DEngine()->m_pObjectsTree[nSID])
							m_arrUpdateStreamingPrioriryStack.Add(Get3DEngine()->m_pObjectsTree[nSID]);
				}
      }

      {
				// Time-sliced scene streaming priority update
				// Update scene faster if in zoom and if camera moving fast
				float fMaxTimeToSpendMS = GetCVars()->e_StreamPredictionUpdateTimeSlice * max(Get3DEngine()->GetAverageCameraSpeed()*.5f, 1.f) / max(m_fZoomFactor, 0.1f);
				fMaxTimeToSpendMS = min(fMaxTimeToSpendMS, GetCVars()->e_StreamPredictionUpdateTimeSlice*2.f);

				CTimeValue maxTimeToSpend;
				maxTimeToSpend.SetSeconds(fMaxTimeToSpendMS*0.001f);

#if !defined(__SPU__)
				const CTimeValue startTime = GetTimer()->GetAsyncTime();
#endif 

				const float fMinDist = bSyncLoad ? 0.f : GetFloatCVar(e_StreamPredictionMinFarZoneDistance);

				const float fMaxViewDistance = Get3DEngine()->GetMaxViewDistance();
        while(m_arrUpdateStreamingPrioriryStack.Count())
        {    
					FRAME_PROFILER( "UpdateObjectsStreamingPriority_MarkNodes", GetSystem(), PROFILE_3DENGINE );

          COctreeNode * pLast = m_arrUpdateStreamingPrioriryStack.Last();
          m_arrUpdateStreamingPrioriryStack.DeleteLast();
					pLast->UpdateStreamingPrioriry(m_arrUpdateStreamingPrioriryStack, fMinDist, fMaxViewDistance, false, m_vStreamPreCachePoint);

					if(
						!bSyncLoad 
#if !defined(__SPU__)
						&& (GetTimer()->GetAsyncTime() - startTime) > maxTimeToSpend
#endif
						)
						break;
        }
      }
    }
  }
  else
  {
    FRAME_PROFILER( "UpdateObjectsStreamingPriority_Mark_NEAR_Nodes", GetSystem(), PROFILE_3DENGINE );

		m_nUpdateStreamingPrioriryRoundIdFast ++;

    PodArray<COctreeNode*> arrUpdateStreamingPrioriryStack;
		const int nVisAreaRecursion = min(GetCVars()->e_StreamPredictionMaxVisAreaRecursion, 2);

		if(CVisArea * pRoot0 = pCurArea)
		{
			m_tmpAreas0.Clear();
			pRoot0->AddConnectedAreas(m_tmpAreas0, nVisAreaRecursion);

			bool bFoundOutside = false;

			if (!GetCVars()->e_StreamPredictionAlwaysIncludeOutside)
			{
				for(int v=0; v<m_tmpAreas0.Count(); v++)
				{
					CVisArea * pN1 = m_tmpAreas0[v];
					if (pN1->IsPortal() && pN1->m_lstConnections.Count() == 1)
					{
						bFoundOutside = true;
						break;
					}
				}
			}
			else
			{
				bFoundOutside = true;
			}

			if (bFoundOutside)
			{
				for(int nSID=0; nSID<Get3DEngine()->m_pObjectsTree.Count(); nSID++)
		      if(Get3DEngine()->IsSegmentSafeToUse(nSID) && Get3DEngine()->m_pObjectsTree[nSID])
						arrUpdateStreamingPrioriryStack.Add(Get3DEngine()->m_pObjectsTree[nSID]);
			}

			for(int v=0; v<m_tmpAreas0.Count(); v++)
			{
				CVisArea * pN1 = m_tmpAreas0[v];
				assert(arrUpdateStreamingPrioriryStack.Find(pN1->m_pObjectsTree)<0);
				if(pN1->m_pObjectsTree)
					arrUpdateStreamingPrioriryStack.Add(pN1->m_pObjectsTree);
			}
		}
		else if(GetVisAreaManager())
		{
			for(int nSID=0; nSID<Get3DEngine()->m_pObjectsTree.Count(); nSID++)
	      if(Get3DEngine()->IsSegmentSafeToUse(nSID) && Get3DEngine()->m_pObjectsTree[nSID])
					arrUpdateStreamingPrioriryStack.Add(Get3DEngine()->m_pObjectsTree[nSID]);
								
			// find portals around
			m_tmpAreas0.Clear();
			GetVisAreaManager()->MakeActiveEntransePortalsList(NULL, m_tmpAreas0, NULL);

			// make list of areas for streaming
			m_tmpAreas1.Clear();
			for(int p=0; p<m_tmpAreas0.Count(); p++)
				if(CVisArea * pRoot = m_tmpAreas0[p])
					pRoot->AddConnectedAreas(m_tmpAreas1, nVisAreaRecursion);

			// fill list of object trees
			for(int v=0; v<m_tmpAreas1.Count(); v++)
			{
				CVisArea * pN1 = m_tmpAreas1[v];
				assert(arrUpdateStreamingPrioriryStack.Find(pN1->m_pObjectsTree)<0);
				if(pN1->m_pObjectsTree)
					arrUpdateStreamingPrioriryStack.Add(pN1->m_pObjectsTree);
			}
		}
		else
		{
			for(int nSID=0; nSID<Get3DEngine()->m_pObjectsTree.Count(); nSID++)
	      if(Get3DEngine()->IsSegmentSafeToUse(nSID) && Get3DEngine()->m_pObjectsTree[nSID])
					arrUpdateStreamingPrioriryStack.Add(Get3DEngine()->m_pObjectsTree[nSID]);
		}

		const float fMaxDist = max(0.f, GetFloatCVar(e_StreamCgfFastUpdateMaxDistance)-GetFloatCVar(e_StreamPredictionDistanceFar));

    while(arrUpdateStreamingPrioriryStack.Count())
    {    
      COctreeNode * pLast = arrUpdateStreamingPrioriryStack.Last();
      arrUpdateStreamingPrioriryStack.DeleteLast();
      pLast->UpdateStreamingPrioriry(arrUpdateStreamingPrioriryStack, 0.f, fMaxDist, true, m_vStreamPreCachePoint);
    }
  }
}

static bool AreTexturesStreamed(IMaterial* pMaterial, float fMipFactor)
{
	// check texture streaming distances
	if(pMaterial)
	{
		if(IRenderShaderResources * pRes = pMaterial->GetShaderItem().m_pShaderResources)
		{
			for(int iSlot = 0;iSlot<EFTT_MAX;++iSlot)
			{
				if(SEfResTexture* pResTex = pRes->GetTexture(iSlot))
				{
					if(ITexture * pITex = pResTex->m_Sampler.m_pITex)
					{
						float fCurMipFactor = fMipFactor * pResTex->GetTiling(0) * pResTex->GetTiling(1);

						if(!pITex->IsParticularMipStreamed(fCurMipFactor))
							return false;
					}
				}
			}
		}
	}

	return true;
}

void CObjManager::CheckTextureReadyFlag()
{
  FUNCTION_PROFILER_3DENGINE;

  if(!m_lstStaticTypes.size())
    return;

  static uint32 nSID = 0;
  static uint32 nGroupId = 0;

  PodArray<StatInstGroup> &rGroupTable = m_lstStaticTypes[nSID];

  if(nGroupId>=rGroupTable.size())
  {
    nGroupId=0;
	nSID++;
  }

  if(nSID>=m_lstStaticTypes.size())
	  nSID=0;

	for (size_t currentGroup = 0; currentGroup < rGroupTable.size(); currentGroup++)
	{
		StatInstGroup & rGroup = rGroupTable[currentGroup];

		if(CStatObj * pStatObj = rGroup.GetStatObj())
		{
			for(int j=0;j<FAR_TEX_COUNT;++j)
			{
				SVegetationSpriteLightInfo& rLightInfo = rGroup.m_arrSSpriteLightInfo[j];
				if(rLightInfo.m_pDynTexture && rLightInfo.m_pDynTexture->GetFlags() & IDynTexture::fNeedRegenerate)
				{
					IMaterial* pMaterial = rGroup.pMaterial;
					if(pMaterial == NULL)
						pMaterial = pStatObj->GetMaterial();

					IRenderMesh *pRenderMesh = pStatObj->GetRenderMesh();
					bool texturesStreamedIn = true;

					if (pRenderMesh)
					{
						PodArray<CRenderChunk> *pChunks = &pRenderMesh->GetChunks();

						if (pChunks)
						{
							for (size_t i = 0; i < pChunks->size(); i++)
							{
								CRenderChunk &currentChunk = (*pChunks)[i];
								IMaterial *pCurrentSubMtl = pMaterial->GetSafeSubMtl(currentChunk.m_nMatID);
								float fMipFactor = rLightInfo.m_MipFactor*currentChunk.m_texelAreaDensity;

								assert(pCurrentSubMtl);
								pCurrentSubMtl->RequestTexturesLoading(fMipFactor);

								if (currentGroup == nGroupId)
								{
									if (!AreTexturesStreamed(pCurrentSubMtl,fMipFactor))
									{
										texturesStreamedIn = false;
									}
								}
							}
						}
					}

					if (currentGroup == nGroupId)
					{
						rGroup.nTexturesAreStreamedIn = texturesStreamedIn;
					}
				}
			}
		}
	}

  nGroupId++;
}

void CObjManager::ProcessObjectsStreaming()
{
  FUNCTION_PROFILER_3DENGINE;

  IF(!GetCVars()->e_StreamCgf,0)
    return;

  // this assert is most likely triggered by forgetting to call 
	// 3dEngine::SyncProcessStreamingUpdate at the end of the frame, leading to multiple
	// updates, but not starting and/or stopping streaming tasks
	assert( m_bNeedProcessObjectsStreaming_Finish == false );
	if( m_bNeedProcessObjectsStreaming_Finish == true )
	{
		CryWarning(VALIDATOR_MODULE_3DENGINE, VALIDATOR_ERROR, "ProcessObjectsStreaming invoked without a following ProcessObjectsStreaming_Finish, please check your update logic");
	}

  float fTimeStart = GetTimer()->GetAsyncCurTime();

  bool bSyncLoad = Get3DEngine()->IsStatObjSyncLoad();

	if(!m_bPreCaheAheadRequested)
	{
		if(GetCamera().GetPosition().GetDistance(m_vStreamPreCachePoint) >= GetFloatCVar(e_StreamCgfGridUpdateDistance))
		{
			Vec3 vOffset = Get3DEngine()->GetAverageCameraMoveDir()*GetFloatCVar(e_StreamPredictionAhead);
			vOffset.z *= .5f;
			m_vStreamPreCachePoint = GetCamera().GetPosition() + vOffset;

			ray_hit hit;
			int rayFlags = geom_colltype_player<<rwi_colltype_bit | rwi_stop_at_pierceable;
			if (m_pPhysicalWorld->RayWorldIntersection(GetCamera().GetPosition(), vOffset, 
				ent_terrain|ent_static|ent_sleeping_rigid|ent_rigid, rayFlags, &hit, 1))
			{
				m_vStreamPreCachePoint = hit.pt;
			}

			if(GetFloatCVar(e_StreamPredictionAheadDebug))
				DrawSphere(m_vStreamPreCachePoint,0.5f);
		}
	}

	UpdateObjectsStreamingPriority(m_bPreCaheAheadRequested || Get3DEngine()->IsContentPrecacheRequested() || bSyncLoad || (GetCVars()->e_StreamCgf==3) || (GetCVars()->e_StreamCgfDebugHeatMap!=0));

	m_bPreCaheAheadRequested = false;

  if(bSyncLoad)
    gEnv->pCryPak->GetFileReadSequencer()->UpdateCurrentThread(true);

  if(bSyncLoad && Get3DEngine()->IsShadersSyncLoad())
    PrintMessage("Pre-caching render meshes, shaders and textures for camera position (%d,%d,%d)", 
    (int)m_vStreamPreCachePoint.x, (int)m_vStreamPreCachePoint.y, (int)m_vStreamPreCachePoint.z);
  else if(bSyncLoad)
    PrintMessage("Pre-caching render meshes for camera position (%d,%d,%d)", 
    (int)m_vStreamPreCachePoint.x, (int)m_vStreamPreCachePoint.y, (int)m_vStreamPreCachePoint.z);

	m_bNeedProcessObjectsStreaming_Finish = true;
	ProcessObjectsStreaming_Impl(bSyncLoad); 
	

	// during precache don't run asynchrony and sync directly to ensure the ESYSTEM_EVENT_LEVEL_PRECACHED
	// event is send to activate the renderthread
	// this also applies to the editor, since it calls the render function multiple times and thus invoking the
	// function multiple times without syncing
	if( bSyncLoad || gEnv->IsEditor() )
	{
		ProcessObjectsStreaming_Finish();
	}

  if(bSyncLoad)
  {
    PrintMessage("Finished pre-caching camera position (%d,%d,%d) in %.1f sec", 
      (int)m_vStreamPreCachePoint.x, (int)m_vStreamPreCachePoint.y, (int)m_vStreamPreCachePoint.z, GetTimer()->GetAsyncCurTime()-fTimeStart);

    gEnv->pCryPak->GetFileReadSequencer()->UpdateCurrentThread();

//		GetRenderer()->RequestFlushAllPendingTextureStreamingJobs(50);
  }
}

void CObjManager::ProcessObjectsStreaming_Impl(bool bSyncLoad)
{
	ProcessObjectsStreaming_Sort(bSyncLoad);
	ProcessObjectsStreaming_Release();
	ProcessObjectsStreaming_InitLoad(bSyncLoad);
}

SPU_NO_INLINE void CObjManager::ProcessObjectsStreaming_Sort(bool bSyncLoad)
{
	int nNumStreamableObjects = m_arrStreamableObjects.Count();

	static float fLastTime = 0;
	const float fTime = GetTimer()->GetAsyncCurTime();

	// call sort only every 100 ms
	if(nNumStreamableObjects && ((fTime > fLastTime + 0.1f) || bSyncLoad))
	{
		FRAME_PROFILER( "ProcessObjectsStreaming_Sort", GetSystem(), PROFILE_3DENGINE );

	  SStreamAbleObject* arrStreamableObjects = &m_arrStreamableObjects[0];
	  assert(arrStreamableObjects);

    const float fMeshStreamingMaxIImportance = 10.f;

    if(bSyncLoad)
    {
      // just put file offset into importance
      for(int i=0; i<nNumStreamableObjects; i++)
      {
        SStreamAbleObject &rObj = arrStreamableObjects[i];

        if(!((CStatObj*)rObj.GetStreamAbleObject())->IsUnloadable())
        {
          rObj.fCurImportance = fMeshStreamingMaxIImportance;
          continue;
        }

        string fileName;
        rObj.GetStreamAbleObject()->GetStreamableName(fileName);
        int nOffset = (int)(GetPak()->GetFileOffsetOnMedia(fileName.c_str())/1024);
        rObj.fCurImportance = -(float)nOffset;
      }
    }
    else
    {
		  for(int i=0; i<nNumStreamableObjects; i++)
		  {
			  SStreamAbleObject &rObj = arrStreamableObjects[i];

        if(!((CStatObj*)rObj.GetStreamAbleObject())->IsUnloadable())
        {
          rObj.fCurImportance = fMeshStreamingMaxIImportance;
          continue;
        }

			  // use data of previous prediction round since current round is not finished yet
			  int nRoundId = CObjManager::m_nUpdateStreamingPrioriryRoundId - 1;

			  // compute importance of objects for selected nRoundId
			  rObj.fCurImportance = -1000.f;

        IStreamable::SInstancePriorityInfo * pInfo = &(rObj.GetStreamAbleObject()->m_arrUpdateStreamingPrioriryRoundInfo[0]);

			  for(int nRoundIdx=0; nRoundIdx<2; nRoundIdx++)
			  {
				  if(pInfo[nRoundIdx].nRoundId == nRoundId)
				  {
					  rObj.fCurImportance = pInfo[nRoundIdx].fMaxImportance;
					  if(rObj.GetLastDrawMainFrameId() > (C3DEngine::GetMainFrameID() - GetCVars()->e_RNTmpDataPoolMaxFrames))
						  rObj.fCurImportance += GetFloatCVar(e_StreamCgfVisObjPriority);
					  break;
				  }
			  }
		  }
    }

		std::sort( SPU_LOCAL_PTR(&arrStreamableObjects[0]), SPU_LOCAL_PTR(&arrStreamableObjects[nNumStreamableObjects]), CObjManager_Cmp_Streamable_Priority() );	
	
		fLastTime = fTime;
	}
}

SPU_NO_INLINE void CObjManager::ProcessObjectsStreaming_Release()
{
	FRAME_PROFILER( "ProcessObjectsStreaming_Release", GetSystem(), PROFILE_3DENGINE ); 
	int nMemoryUsage = 0;

	int nNumStreamableObjects = m_arrStreamableObjects.Count();
	SStreamAbleObject* arrStreamableObjects =nNumStreamableObjects ? &m_arrStreamableObjects[0] :NULL;
	
	for(int nObjId=0; nObjId<nNumStreamableObjects; nObjId++)
	{
		const SStreamAbleObject &rObj = arrStreamableObjects[nObjId];

		nMemoryUsage += rObj.GetStreamableContentMemoryUsage();

		bool bUnload = nMemoryUsage >= GetCVars()->e_StreamCgfPoolSize*1024*1024;

		if(!bUnload && GetCVars()->e_StreamCgfDebug==4)
			if(rObj.GetStreamAbleObject()->m_arrUpdateStreamingPrioriryRoundInfo[0].nRoundId < (CObjManager::m_nUpdateStreamingPrioriryRoundId-8))
				bUnload = true;

		if(bUnload && ((CStatObj*)rObj.GetStreamAbleObject())->IsUnloadable())
		{
			if(rObj.GetStreamAbleObject()->m_eStreamingStatus == ecss_Ready)
			{
				m_arrStreamableToRelease.push(rObj.GetStreamAbleObject());
			}  

			// remove from list if not active for long time
			if(rObj.GetStreamAbleObject()->m_eStreamingStatus == ecss_NotLoaded)
			{
				if(rObj.GetStreamAbleObject()->m_arrUpdateStreamingPrioriryRoundInfo[0].nRoundId < (CObjManager::m_nUpdateStreamingPrioriryRoundId-8))
				{
					rObj.GetStreamAbleObject()->m_arrUpdateStreamingPrioriryRoundInfo[0].nRoundId = 0;
					m_arrStreamableToDelete.push(rObj.GetStreamAbleObject());
				}
			}
		}
	}
	s_nLastStreamingMemoryUsage = nMemoryUsage;
}

SPU_NO_INLINE void CObjManager::ProcessObjectsStreaming_InitLoad(bool bSyncLoad)
{
	FRAME_PROFILER( "ProcessObjectsStreaming_InitLoad", GetSystem(), PROFILE_3DENGINE );

  int nMaxMemUsage = GetCVars()->e_StreamCgfPoolSize*1024*1024; 
	int nNumStreamableObjects = m_arrStreamableObjects.Count();
	SStreamAbleObject* arrStreamableObjects = nNumStreamableObjects ? &m_arrStreamableObjects[0] :NULL;

  int nMemoryUsage = 0;
  
  for(int nObjId = 0; nObjId<nNumStreamableObjects; nObjId++)
  {
    const SStreamAbleObject &rObj = arrStreamableObjects[nObjId];
    
    int size = rObj.GetStreamableContentMemoryUsage();
    nMemoryUsage += size; 
    if(nMemoryUsage >= nMaxMemUsage)
      break;

    if(rObj.GetStreamAbleObject()->m_eStreamingStatus == ecss_NotLoaded)
    {
      m_arrStreamableToLoad.push( rObj.GetStreamAbleObject() );				
	
      if(!bSyncLoad)
        break;

      if((GetCVars()->e_AutoPrecacheCgf == 2) && (nObjId > nNumStreamableObjects/2))
        break;
    }
  }
}

void CObjManager::ProcessObjectsStreaming_Finish()
{
	if( m_bNeedProcessObjectsStreaming_Finish == false )
		return;

	LOADING_TIME_PROFILE_SECTION;

	int nMaxInProgress = GetCVars()->e_StreamCgfMaxTasksInProgress;
	int nNumStreamableObjects = m_arrStreamableObjects.Count();
  int nInProgress=0;
  int nInProgressMem = 0;
	m_bNeedProcessObjectsStreaming_Finish = false;

  SMeshPoolStatistics stats;
  GetRenderer()->EF_Query(EFQ_GetMeshPoolInfo, (INT_PTR)&stats);
	
	FRAME_PROFILER( "ProcessObjectsStreaming_Finish", GetSystem(), PROFILE_3DENGINE );
	bool bSyncLoad = Get3DEngine()->IsStatObjSyncLoad();	
	
	{
		LOADING_TIME_PROFILE_SECTION;


    bool bUnloaded = false;
    // now unload the stat object
    while( !m_arrStreamableToRelease.empty() )
    {
      bUnloaded = true;
      IStreamable *pStatObj = m_arrStreamableToRelease.pop();
      pStatObj->ReleaseStreamableContent();		

      if(GetCVars()->e_StreamCgfDebug==2)
      {
        string sName;
        pStatObj->GetStreamableName(sName);
        PrintMessage("Unloaded: %s", sName.c_str());
      }
    }
	
    for(int nObjId = 0; nObjId<nNumStreamableObjects; nObjId++)
    {
      const SStreamAbleObject &rObj = m_arrStreamableObjects[nObjId];
      if(rObj.GetStreamAbleObject()->m_eStreamingStatus == ecss_InProgress)
        nInProgress++;
    }

    int nSyncObjCounter = 0;

    // start streaming of stat objects
    if (bSyncLoad) 
      gEnv->pRenderer->EnableBatchMode(true);

    //const uint nMaxNumberOfSyncStreamingRequests = GetCVars()->e_AutoPrecacheCgfMaxTasks;

    std::vector<IReadStreamPtr> streamsToFinish;

		GetISystem()->GetStreamEngine()->BeginReadGroup(); // Make sure new streaming requests are accumulated

    for( int nId = 0; (nId<m_arrStreamableToLoad.size()) && (nInProgress < nMaxInProgress || bSyncLoad); nId++ )
    {
      IStreamable *pStatObj = m_arrStreamableToLoad[nId];

      // If there is a rendermesh pool present, only submit streaming job if
      // the pool size in the renderer is sufficient. NOTE: This is a weak
      // test. Better test would be to actually preallocate the memory here and
      // only submit the streaming job if the memory could be obtained. 
      int size = pStatObj->GetStreamableContentMemoryUsage();
			size_t poolLimit = stats.nPoolSize<<2; // 4 times the pool limit because the rendermesh pool has been reduced
      if (poolLimit && poolLimit <= (size_t)(nInProgressMem + size))
      {
        // Actually the above check is an implicit size limit on CGFs - which is awful because it can lead to meshes never 
        // streamed in. This has to change after crysis2 - for now, this will include a white list of meshes that need to be 
        // loaded for crysis2 in the prism2 level. 
        bool skip = true; 
        if ((size_t)poolLimit <= (size_t)size)
        {
          CStatObj* cpStatObj = (CStatObj*)pStatObj;
          if (const char* szFileName = cpStatObj->m_szFileName.c_str()) 
          {
            CryLogAlways("[WARNING] statobj '%s' skipped because too large (%d kb (>= %d kb limit))", szFileName, (int)(size/1024), (int)(stats.nPoolSize/1024)); 
          }
        }
        if (skip) 
          continue; 
      }
      nInProgressMem += size; 
      
      if(GetCVars()->e_StreamCgfDebug==2)
      {
        string sName;
        pStatObj->GetStreamableName(sName);
        PrintMessage("Loading: %s", sName.c_str());
      }

      IReadStreamPtr stream;
      pStatObj->StartStreaming( bSyncLoad, bSyncLoad?&stream:NULL );
      ++nInProgress;

      if(bSyncLoad && stream)
        streamsToFinish.push_back(stream);

			/*
      if(streamsToFinish.size() >= nMaxNumberOfSyncStreamingRequests)
      {
        streamsToFinish[0]->Wait();
        streamsToFinish.erase(streamsToFinish.begin());
        nSyncObjCounter++;
      }
			*/
    }

		GetISystem()->GetStreamEngine()->EndReadGroup(); // Make sure new streaming requests are accumulated

    while(!streamsToFinish.empty())
    {
      streamsToFinish.back()->Wait( 5000 ); // Wait for maximum of 5 seconds (can prevent super rare freeze)
      streamsToFinish.pop_back();
      nSyncObjCounter++;
    }

    if (bSyncLoad) 
      gEnv->pRenderer->EnableBatchMode(false);

    if(bSyncLoad)
      PrintMessage("Finished synchronous pre-cache of render meshes for %d CGF's", nSyncObjCounter);

    m_arrStreamableToLoad.clear();
	
    // remove no longer needed objects from list
    while( !m_arrStreamableToDelete.empty() )
    {
      IStreamable *pStatObj = m_arrStreamableToDelete.pop();
      SStreamAbleObject stramAbleObject(pStatObj);
      m_arrStreamableObjects.Delete(stramAbleObject);
    }
	}
}

void CObjManager::GetObjectsStreamingStatus(I3DEngine::SObjectsStreamingStatus &outStatus)
{
	outStatus.nReady = outStatus.nInProgress = outStatus.nTotal = outStatus.nAllocatedBytes = outStatus.nMemRequired = 0;
	outStatus.nMeshPoolSize = GetCVars()->e_StreamCgfPoolSize;

  for (LoadedObjects::iterator it = m_lstLoadedObjects.begin(); it != m_lstLoadedObjects.end(); ++it)
  {
    CStatObj *pStatObj = *it;
    if(pStatObj->m_bSubObject)
      continue;

    if(pStatObj->m_pLod0)
      continue;

    for(int l=0; l<MAX_STATOBJ_LODS_NUM; l++)
      if(CStatObj * pLod = (CStatObj *)pStatObj->GetLodObject(l))
        outStatus.nTotal++;
  }

  outStatus.nActive = m_arrStreamableObjects.Count();

  for(int nObjId = 0; nObjId<m_arrStreamableObjects.Count(); nObjId++)
  {
    const SStreamAbleObject &rStreamAbleObject = m_arrStreamableObjects[nObjId];

    if(rStreamAbleObject.GetStreamAbleObject()->m_eStreamingStatus == ecss_Ready)
      outStatus.nReady++;
    
    if(rStreamAbleObject.GetStreamAbleObject()->m_eStreamingStatus == ecss_InProgress)
      outStatus.nInProgress++;

    if(rStreamAbleObject.GetStreamAbleObject()->m_eStreamingStatus == ecss_Ready)
      outStatus.nAllocatedBytes += rStreamAbleObject.GetStreamableContentMemoryUsage();

    if(rStreamAbleObject.GetStreamAbleObject()->m_arrUpdateStreamingPrioriryRoundInfo[0].nRoundId >= (CObjManager::m_nUpdateStreamingPrioriryRoundId-4))
      outStatus.nMemRequired += rStreamAbleObject.GetStreamableContentMemoryUsage();
  }
}

void CObjManager::PrecacheStatObjMaterial( IMaterial* pMaterial, const float fEntDistance, IStatObj *pStatObj, bool bFullUpdate, bool bDrawNear )
{
//  FUNCTION_PROFILER_3DENGINE;

	if (pStatObj)
	{
    if(!pMaterial)
      pMaterial = pStatObj->GetMaterial();

		for (int i = 0; i < pStatObj->GetSubObjectCount(); i++)
		{
			PrecacheStatObjMaterial(pMaterial,fEntDistance,pStatObj->GetSubObject(i)->pStatObj,bFullUpdate,bDrawNear);
		}

		if(CMatInfo * pMatInfo = (CMatInfo *)pMaterial)
			pMatInfo->PrecacheMaterial(fEntDistance, pStatObj->GetRenderMesh(), bFullUpdate, bDrawNear);
	}
}

void CObjManager::PrecacheCharacter(IRenderNode * pObj, const float fImportance,  ICharacterInstance * pCharacter, IMaterial* pSlotMat, const Matrix34& matParent, const float fEntDistance, const float fScale, int nMaxDepth, bool bFullUpdate, bool bDrawNear, int nLod)
{
  if(!nMaxDepth || !pCharacter)
    return;
  nMaxDepth--;

	if(IAttachmentManager * pAttMan = pCharacter->GetIAttachmentManager())
  {
		if(ICharacterInstance *pCharInstance = pAttMan->GetSkinInstance())
		{
			if(pCharInstance != pCharacter)
			{
				PrecacheCharacter(pObj, fImportance, pCharInstance, pSlotMat, matParent, fEntDistance, fScale, nMaxDepth, bFullUpdate, bDrawNear, nLod);
			}
		}
		if(ICharacterInstance *pCharInstance = pAttMan->GetSkelInstance())
		{
			if(pCharInstance != pCharacter)
			{
				PrecacheCharacter(pObj, fImportance, pCharInstance, pSlotMat, matParent, fEntDistance, fScale, nMaxDepth, bFullUpdate, bDrawNear, nLod);
			}
		}

    int nCount = pAttMan->GetAttachmentCount();
    for(int i=0; i<nCount; i++)
    {
      if(IAttachment * pAtt = pAttMan->GetInterfaceByIndex(i))
      {
        if(IAttachmentObject * pAttObj = pAtt->GetIAttachmentObject())
        {
					IMaterial* pAttMatOverride = pAttObj->GetMaterialOverride();
					IMaterial* pAttMat = pAttMatOverride ? pAttMatOverride : pAttObj->GetMaterial();
          if(ICharacterInstance *pCharInstance = pAttObj->GetICharacterInstance())
						PrecacheCharacter(pObj, fImportance, pCharInstance, pAttMat, matParent, fEntDistance, fScale, nMaxDepth, bFullUpdate, bDrawNear, nLod);

          if(CStatObj * pStatObj = (CStatObj *)pAttObj->GetIStatObj())
          {
            if(!pStatObj || pStatObj->GetFlags()&STATIC_OBJECT_HIDDEN)
              continue;

            const QuatT q = pAtt->GetAttAbsoluteDefault();

            Matrix34A tm34 = matParent*Matrix34(q);
            pStatObj->UpdateStreamableComponents(fImportance, tm34, pObj, fEntDistance, bFullUpdate, nLod);

						IMaterial* pStatObjMat = pStatObj->GetMaterial();
						PrecacheStatObjMaterial(pAttMat ? pAttMat : pStatObjMat, fEntDistance / fScale, pStatObj, bFullUpdate, bDrawNear);
          }
        }
      }
    }

		IMaterial* pCharObjMat = pCharacter->GetMaterial();

		IRenderMesh *pRenderMesh = pCharacter->GetRenderMesh();

		if(pSlotMat)
			((CMatInfo*)pSlotMat)->PrecacheMaterial(fEntDistance / fScale, pRenderMesh, bFullUpdate, bDrawNear);

		if(pCharObjMat)
			((CMatInfo*)pCharObjMat)->PrecacheMaterial(fEntDistance / fScale, pRenderMesh, bFullUpdate, bDrawNear);
  }

	// joints
  if(ISkeletonPose * pSkeletonPose = pCharacter->GetISkeletonPose())
	{
		uint32 numJoints = pSkeletonPose->GetJointCount();

		// check StatObj attachments
		for(uint32 i=0; i<numJoints; i++)
		{
			CStatObj * pStatObj = (CStatObj *)pSkeletonPose->GetStatObjOnJoint(i);
			if(!pStatObj || pStatObj->GetFlags()&STATIC_OBJECT_HIDDEN)
				continue;

			Matrix34A tm34 = matParent*Matrix34(pSkeletonPose->GetAbsJointByID(i));
			pStatObj->UpdateStreamableComponents(fImportance, tm34, pObj, fEntDistance, bFullUpdate, nLod);

			IMaterial* pStatObjMat = pStatObj->GetMaterial();
			PrecacheStatObjMaterial(pSlotMat ? pSlotMat : pStatObjMat, fEntDistance / fScale, pStatObj, bFullUpdate, bDrawNear);
		}
	}
}

void CObjManager::UpdateRenderNodeStreamingPrioriry(IRenderNode * pObj, float fEntDistanceReal, bool bFullUpdate)
{
//  FUNCTION_PROFILER_3DENGINE;

  if(pObj->m_dwRndFlags&ERF_HIDDEN)
    return;

  if(pObj->m_fWSMaxViewDist<0.01f)
    return;

  if( pObj->GetIntegrationType() && GetCVars()->e_VoxTerHideIntegrated && Get3DEngine()->GetIVoxTerrain() )
    return; // baked into terrain

  // check cvars
	const	EERType nodeType = pObj->GetRenderNodeType();

	const float fEntDistance = max(0.f, fEntDistanceReal - GetFloatCVar(e_StreamPredictionDistanceNear));

	float fImportance = 1.f - (fEntDistance / ((pObj->m_fWSMaxViewDist + GetFloatCVar(e_StreamPredictionDistanceFar))));

	if(fImportance<0)
		return;

	float fObjScale = 1.f;
	AABB objBox(pObj->GetBBox());

  switch(nodeType)
  {
  case eERType_Brush:
    if(!GetCVars()->e_Brushes) return;
		fObjScale = max(0.001f,((CBrush *)pObj)->CBrush::GetMatrix().GetColumn0().GetLength());
    break;
  case eERType_Vegetation:
    if(!GetCVars()->e_Vegetation) return;
		fObjScale = max(0.001f,((CVegetation *)pObj)->CVegetation::GetScale());
		fObjScale /= max(0.01f,((CVegetation *)pObj)->CVegetation::GetLodRatioNormalized());
    break;
	case eERType_ParticleEmitter:
		if(!GetCVars()->e_Particles) return;
    break;
  case eERType_Decal:
    if(!GetCVars()->e_Decals) return;
		fObjScale = max(0.001f,((CDecalRenderNode *)pObj)->CDecalRenderNode::GetMatrix().GetColumn0().GetLength());
    break;
  case eERType_VoxelObject:
    if(!GetCVars()->e_Voxel) return;
    break;
  case eERType_WaterWave:
    if(!GetCVars()->e_WaterWaves) return;
    break;
	case eERType_WaterVolume:
		if(!GetCVars()->e_WaterVolumes) return;
		break;
  case eERType_Road:
    if(!GetCVars()->e_Roads) return;
		fObjScale = max(0.001f,((CRoadRenderNode*)pObj)->m_arrVerts[0].GetDistance(((CRoadRenderNode*)pObj)->m_arrVerts[1]));
    break;
	case eERType_Light:
		if(!GetCVars()->e_DynamicLights) return;
		break;
	case eERType_FogVolume:
		fObjScale = max(0.001f,((CFogVolumeRenderNode *)pObj)->CFogVolumeRenderNode::GetMatrix().GetColumn0().GetLength());
		break;
	case eERType_Cloud:
	case eERType_DistanceCloud:
		fObjScale = max(0.001f,pObj->GetBBox().GetRadius());
  default:
    if(!GetCVars()->e_Entities) return;
    break;
  }

	float fInvObjScale = 1.0f / fObjScale;

	int nLod = CObjManager::GetObjectLOD(fEntDistanceReal, pObj->GetLodRatioNormalized(), objBox.GetRadius());

	IMaterial* pRenderNodeMat = pObj->GetMaterialOverride();
	{
		IStatObj *pStatObj = pObj->GetEntityStatObj();
		if (pStatObj != NULL)
		{
			pStatObj = pStatObj->GetLodObject(nLod, true);
			PrecacheStatObjMaterial(pRenderNodeMat, fEntDistance * fInvObjScale, pStatObj, bFullUpdate, false);
		}
		else
		{
			if(CMatInfo * pMatInfo = (CMatInfo *)pRenderNodeMat) {
				pMatInfo->PrecacheMaterial(fEntDistance * fInvObjScale, pObj->GetRenderMesh(nLod), bFullUpdate, false);
			}
		}
	}

  if(nodeType == eERType_ParticleEmitter)
  {
    IParticleEmitter* pEmitter = (IParticleEmitter*)pObj;
    Matrix34A tm34A; tm34A.SetIdentity();
    tm34A.SetTranslation(pEmitter->GetPos());
    pEmitter->UpdateStreamableComponents(fImportance, tm34A, pObj, fEntDistance, bFullUpdate, nLod);
    return;
  }

  if(pObj->m_pRNTmpData)
  {
		if(GetFloatCVar(e_StreamCgfGridUpdateDistance) || GetFloatCVar(e_StreamPredictionAhead) || GetFloatCVar(e_StreamPredictionMinFarZoneDistance))
		{
			float fDistanceToCam = cry_sqrtf(Distance::Point_AABBSq(GetCamera().GetPosition(),objBox))*m_fZoomFactor;
			pObj->m_pRNTmpData->userData.nLod = CObjManager::GetObjectLOD(fDistanceToCam, pObj->GetLodRatioNormalized(), objBox.GetRadius());
		}
		else
			pObj->m_pRNTmpData->userData.nLod = nLod;

    if(pObj->GetRndFlags()&ERF_OUTDOORONLY && pObj->m_pRNTmpData->userData.pRenderObject)
    {
      ColorF ambColor = Get3DEngine()->GetSkyColor();
      pObj->m_pRNTmpData->userData.pRenderObject->m_II.m_AmbColor.r = ambColor.r;
      pObj->m_pRNTmpData->userData.pRenderObject->m_II.m_AmbColor.g = ambColor.g;
      pObj->m_pRNTmpData->userData.pRenderObject->m_II.m_AmbColor.b = ambColor.b;
    }
		/*
    if(pObj->GetRenderNodeType() == eERType_Vegetation)
    {
      CVegetation*pVeg = (CVegetation*)pObj;
      pVeg->UpdateBending();
    }    
    */
  }

  const int nSlotCount = pObj->GetSlotCount();

  for(int nEntSlot = 0; nEntSlot<nSlotCount; nEntSlot++)
  {
		bool bDrawNear = false;

		IMaterial* pSlotMat = pObj->GetEntitySlotMaterial(nEntSlot, false, &bDrawNear);
		if(!pSlotMat)
			pSlotMat = pRenderNodeMat;

    Matrix34A matParent;

    CStatObj * pStatObj = (CStatObj *)pObj->GetEntityStatObj(nEntSlot, 0, &matParent, false);
    if (pStatObj)
    {
			pStatObj = (CStatObj *)pStatObj->GetLodObject(nLod, true);
      pStatObj->UpdateStreamableComponents(fImportance, matParent, pObj, fEntDistance, bFullUpdate, nLod);			

			IMaterial* pStatObjMat = pStatObj->GetMaterial();
			PrecacheStatObjMaterial(pSlotMat ? pSlotMat : pStatObjMat, fEntDistance * fInvObjScale, pStatObj, bFullUpdate, bDrawNear);
    }

    if(ICharacterInstance * pChar = pObj->GetEntityCharacter(nEntSlot, &matParent, false))
    {
      PrecacheCharacter(pObj, fImportance, pChar, pSlotMat, matParent, fEntDistance, fObjScale, 2, bFullUpdate, bDrawNear, nLod);
    }
  }
}

#undef USE_SPU
